{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import word2vec\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import json\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news-bm.json','r') as fopen:\n",
    "    sentences = json.loads(fopen.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_array, dictionary, rev_dictionary, num_lines, num_words = word2vec.build_word_array(sentences,vocab_size=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = word2vec.build_training_set(word_array)\n",
    "graph_params = {'batch_size': 32,\n",
    "                'vocab_size': np.max(X)+1,\n",
    "                'embed_size': 512,\n",
    "                'hid_size': 512,\n",
    "                'neg_samples': 128,\n",
    "                'learn_rate': 0.01,\n",
    "                'momentum': 0.9,\n",
    "                'embed_noise': 0.1,\n",
    "                'hid_noise': 0.3,\n",
    "                'epoch':500,\n",
    "                'optimizer': 'Momentum'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "split = round(X.shape[0]*0.9)\n",
    "X_val, Y_val = (X[split:, :], Y[split:, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model built, vocab size 33735, document length 1211339\n"
     ]
    }
   ],
   "source": [
    "model = word2vec.Model(graph_params)\n",
    "print('model built, vocab size %d, document length %d'%(np.max(X)+1, len(word_array)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, total batch 37854, train loss 16.309003, val loss 4.358148\n",
      "epoch 2, total batch 41639, train loss 4.276934, val loss 4.225277\n",
      "epoch 3, total batch 45424, train loss 4.152353, val loss 4.105800\n",
      "epoch 4, total batch 49209, train loss 4.038621, val loss 4.008378\n",
      "epoch 5, total batch 52994, train loss 3.955050, val loss 3.908697\n",
      "epoch 6, total batch 56779, train loss 3.858267, val loss 3.836481\n",
      "epoch 7, total batch 60564, train loss 3.774097, val loss 3.756378\n",
      "epoch 8, total batch 64349, train loss 3.704103, val loss 3.693702\n",
      "epoch 9, total batch 68134, train loss 3.634720, val loss 3.628249\n",
      "epoch 10, total batch 71919, train loss 3.573237, val loss 3.564560\n",
      "epoch 11, total batch 75704, train loss 3.514016, val loss 3.506671\n",
      "epoch 12, total batch 79489, train loss 3.457400, val loss 3.444773\n",
      "epoch 13, total batch 83274, train loss 3.399341, val loss 3.411058\n",
      "epoch 14, total batch 87059, train loss 3.341881, val loss 3.356764\n",
      "epoch 15, total batch 90844, train loss 3.301628, val loss 3.296608\n",
      "epoch 16, total batch 94629, train loss 3.250811, val loss 3.268333\n",
      "epoch 17, total batch 98414, train loss 3.198903, val loss 3.213219\n",
      "epoch 18, total batch 102199, train loss 3.155310, val loss 3.181903\n",
      "epoch 19, total batch 105984, train loss 3.124056, val loss 3.139863\n",
      "epoch 20, total batch 109769, train loss 3.081588, val loss 3.097685\n",
      "epoch 21, total batch 113554, train loss 3.037644, val loss 3.061893\n",
      "epoch 22, total batch 117339, train loss 3.000918, val loss 3.040554\n",
      "epoch 23, total batch 121124, train loss 2.966083, val loss 2.992816\n",
      "epoch 24, total batch 124909, train loss 2.929374, val loss 2.954375\n",
      "epoch 25, total batch 128694, train loss 2.884844, val loss 2.918647\n",
      "epoch 26, total batch 132479, train loss 2.868054, val loss 2.878850\n",
      "epoch 27, total batch 136264, train loss 2.826478, val loss 2.854492\n",
      "epoch 28, total batch 140049, train loss 2.799820, val loss 2.826666\n",
      "epoch 29, total batch 143834, train loss 2.757784, val loss 2.808330\n",
      "epoch 30, total batch 147619, train loss 2.733476, val loss 2.775188\n",
      "epoch 31, total batch 151404, train loss 2.705827, val loss 2.741235\n",
      "epoch 32, total batch 155189, train loss 2.676559, val loss 2.719561\n",
      "epoch 33, total batch 158974, train loss 2.644251, val loss 2.685512\n",
      "epoch 34, total batch 162759, train loss 2.615027, val loss 2.664673\n",
      "epoch 35, total batch 166544, train loss 2.589556, val loss 2.638877\n",
      "epoch 36, total batch 170329, train loss 2.562592, val loss 2.610097\n",
      "epoch 37, total batch 174114, train loss 2.533356, val loss 2.590830\n",
      "epoch 38, total batch 177899, train loss 2.523470, val loss 2.567235\n",
      "epoch 39, total batch 181684, train loss 2.500337, val loss 2.532328\n",
      "epoch 40, total batch 185469, train loss 2.465399, val loss 2.514919\n",
      "epoch 41, total batch 189254, train loss 2.439261, val loss 2.488483\n",
      "epoch 42, total batch 193039, train loss 2.417288, val loss 2.470153\n",
      "epoch 43, total batch 196824, train loss 2.386678, val loss 2.446919\n",
      "epoch 44, total batch 200609, train loss 2.373374, val loss 2.433372\n",
      "epoch 45, total batch 204394, train loss 2.333541, val loss 2.399833\n",
      "epoch 46, total batch 208179, train loss 2.314567, val loss 2.382216\n",
      "epoch 47, total batch 211964, train loss 2.293799, val loss 2.366165\n",
      "epoch 48, total batch 215749, train loss 2.287210, val loss 2.340532\n",
      "epoch 49, total batch 219534, train loss 2.257443, val loss 2.334468\n",
      "epoch 50, total batch 223319, train loss 2.240399, val loss 2.305916\n",
      "epoch 51, total batch 227104, train loss 2.220153, val loss 2.277681\n",
      "epoch 52, total batch 230889, train loss 2.192562, val loss 2.258455\n",
      "epoch 53, total batch 234674, train loss 2.168029, val loss 2.240919\n",
      "epoch 54, total batch 238459, train loss 2.158011, val loss 2.228424\n",
      "epoch 55, total batch 242244, train loss 2.125425, val loss 2.202730\n",
      "epoch 56, total batch 246029, train loss 2.111623, val loss 2.185934\n",
      "epoch 57, total batch 249814, train loss 2.090067, val loss 2.164203\n",
      "epoch 58, total batch 253599, train loss 2.073211, val loss 2.153430\n",
      "epoch 59, total batch 257384, train loss 2.062545, val loss 2.132401\n",
      "epoch 60, total batch 261169, train loss 2.035677, val loss 2.109691\n",
      "epoch 61, total batch 264954, train loss 2.020709, val loss 2.093887\n",
      "epoch 62, total batch 268739, train loss 1.997772, val loss 2.077037\n",
      "epoch 63, total batch 272524, train loss 1.989096, val loss 2.061649\n",
      "epoch 64, total batch 276309, train loss 1.964493, val loss 2.037463\n",
      "epoch 65, total batch 280094, train loss 1.957466, val loss 2.028399\n",
      "epoch 66, total batch 283879, train loss 1.935078, val loss 2.007454\n",
      "epoch 67, total batch 287664, train loss 1.906711, val loss 1.994600\n",
      "epoch 68, total batch 291449, train loss 1.899230, val loss 1.973273\n",
      "epoch 69, total batch 295234, train loss 1.874401, val loss 1.960002\n",
      "epoch 70, total batch 299019, train loss 1.854968, val loss 1.947269\n",
      "epoch 71, total batch 302804, train loss 1.846531, val loss 1.922093\n",
      "epoch 72, total batch 306589, train loss 1.832217, val loss 1.919242\n",
      "epoch 73, total batch 310374, train loss 1.809974, val loss 1.896132\n",
      "epoch 74, total batch 314159, train loss 1.793648, val loss 1.889151\n",
      "epoch 75, total batch 317944, train loss 1.790586, val loss 1.865274\n",
      "epoch 76, total batch 321729, train loss 1.767512, val loss 1.856565\n",
      "epoch 77, total batch 325514, train loss 1.747327, val loss 1.842001\n",
      "epoch 78, total batch 329299, train loss 1.745184, val loss 1.820799\n",
      "epoch 79, total batch 333084, train loss 1.723159, val loss 1.806248\n",
      "epoch 80, total batch 336869, train loss 1.697556, val loss 1.798289\n",
      "epoch 81, total batch 340654, train loss 1.696762, val loss 1.776841\n",
      "epoch 82, total batch 344439, train loss 1.672159, val loss 1.759351\n",
      "epoch 83, total batch 348224, train loss 1.662978, val loss 1.747111\n",
      "epoch 84, total batch 352009, train loss 1.650342, val loss 1.736137\n",
      "epoch 85, total batch 355794, train loss 1.637217, val loss 1.722019\n",
      "epoch 86, total batch 359579, train loss 1.629329, val loss 1.697215\n",
      "epoch 87, total batch 363364, train loss 1.604784, val loss 1.690000\n",
      "epoch 88, total batch 367149, train loss 1.586699, val loss 1.682574\n",
      "epoch 89, total batch 370934, train loss 1.580003, val loss 1.658438\n",
      "epoch 90, total batch 374719, train loss 1.562184, val loss 1.650461\n",
      "epoch 91, total batch 378504, train loss 1.557368, val loss 1.644879\n",
      "epoch 92, total batch 382289, train loss 1.545658, val loss 1.623706\n",
      "epoch 93, total batch 386074, train loss 1.525502, val loss 1.613592\n",
      "epoch 94, total batch 389859, train loss 1.513663, val loss 1.596128\n",
      "epoch 95, total batch 393644, train loss 1.498754, val loss 1.583363\n",
      "epoch 96, total batch 397429, train loss 1.488364, val loss 1.575166\n",
      "epoch 97, total batch 401214, train loss 1.481661, val loss 1.559975\n",
      "epoch 98, total batch 404999, train loss 1.461598, val loss 1.535337\n",
      "epoch 99, total batch 408784, train loss 1.451303, val loss 1.528432\n",
      "epoch 100, total batch 412569, train loss 1.444693, val loss 1.523676\n",
      "epoch 101, total batch 416354, train loss 1.426800, val loss 1.516293\n",
      "epoch 102, total batch 420139, train loss 1.415899, val loss 1.498189\n",
      "epoch 103, total batch 423924, train loss 1.402654, val loss 1.486777\n",
      "epoch 104, total batch 427709, train loss 1.389755, val loss 1.484527\n",
      "epoch 105, total batch 431494, train loss 1.385806, val loss 1.461502\n",
      "epoch 106, total batch 435279, train loss 1.373604, val loss 1.447300\n",
      "epoch 107, total batch 439064, train loss 1.355117, val loss 1.430533\n",
      "epoch 108, total batch 442849, train loss 1.342614, val loss 1.425538\n",
      "epoch 109, total batch 446634, train loss 1.337180, val loss 1.419181\n",
      "epoch 110, total batch 450419, train loss 1.323914, val loss 1.407263\n",
      "epoch 111, total batch 454204, train loss 1.310895, val loss 1.399063\n",
      "epoch 112, total batch 457989, train loss 1.309019, val loss 1.393721\n",
      "epoch 113, total batch 461774, train loss 1.295988, val loss 1.378635\n",
      "epoch 114, total batch 465559, train loss 1.286936, val loss 1.370971\n",
      "epoch 115, total batch 469344, train loss 1.268044, val loss 1.350984\n",
      "epoch 116, total batch 473129, train loss 1.265137, val loss 1.349554\n",
      "epoch 117, total batch 476914, train loss 1.252349, val loss 1.336241\n",
      "epoch 118, total batch 480699, train loss 1.246018, val loss 1.326861\n",
      "epoch 119, total batch 484484, train loss 1.227809, val loss 1.307828\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 120, total batch 488269, train loss 1.219081, val loss 1.304290\n",
      "epoch 121, total batch 492054, train loss 1.209840, val loss 1.297749\n",
      "epoch 122, total batch 495839, train loss 1.201316, val loss 1.285423\n",
      "epoch 123, total batch 499624, train loss 1.201723, val loss 1.269846\n",
      "epoch 124, total batch 503409, train loss 1.182213, val loss 1.264103\n",
      "epoch 125, total batch 507194, train loss 1.166812, val loss 1.256118\n",
      "epoch 126, total batch 510979, train loss 1.168572, val loss 1.246695\n",
      "epoch 127, total batch 514764, train loss 1.149207, val loss 1.228159\n",
      "epoch 128, total batch 518549, train loss 1.145762, val loss 1.222112\n",
      "epoch 129, total batch 522334, train loss 1.138128, val loss 1.209894\n",
      "epoch 130, total batch 526119, train loss 1.133577, val loss 1.199129\n",
      "epoch 131, total batch 529904, train loss 1.126865, val loss 1.191883\n",
      "epoch 132, total batch 533689, train loss 1.109273, val loss 1.192349\n",
      "epoch 133, total batch 537474, train loss 1.100040, val loss 1.177717\n",
      "epoch 134, total batch 541259, train loss 1.099630, val loss 1.160509\n",
      "epoch 135, total batch 545044, train loss 1.084989, val loss 1.156345\n",
      "epoch 136, total batch 548829, train loss 1.077156, val loss 1.150513\n",
      "epoch 137, total batch 552614, train loss 1.067509, val loss 1.137477\n",
      "epoch 138, total batch 556399, train loss 1.058521, val loss 1.132648\n",
      "epoch 139, total batch 560184, train loss 1.053905, val loss 1.127626\n",
      "epoch 140, total batch 563969, train loss 1.049340, val loss 1.116183\n",
      "epoch 141, total batch 567754, train loss 1.038948, val loss 1.107458\n",
      "epoch 142, total batch 571539, train loss 1.034201, val loss 1.092521\n",
      "epoch 143, total batch 575324, train loss 1.023342, val loss 1.090416\n",
      "epoch 144, total batch 579109, train loss 1.010733, val loss 1.082350\n",
      "epoch 145, total batch 582894, train loss 1.011072, val loss 1.072753\n",
      "epoch 146, total batch 586679, train loss 0.995102, val loss 1.066333\n",
      "epoch 147, total batch 590464, train loss 0.989383, val loss 1.055577\n",
      "epoch 148, total batch 594249, train loss 0.977414, val loss 1.050558\n",
      "epoch 149, total batch 598034, train loss 0.979141, val loss 1.044780\n",
      "epoch 150, total batch 601819, train loss 0.964313, val loss 1.036016\n",
      "epoch 151, total batch 605604, train loss 0.949648, val loss 1.028741\n",
      "epoch 152, total batch 609389, train loss 0.949263, val loss 1.020521\n",
      "epoch 153, total batch 613174, train loss 0.950953, val loss 1.011782\n",
      "epoch 154, total batch 616959, train loss 0.935397, val loss 1.006661\n",
      "epoch 155, total batch 620744, train loss 0.935194, val loss 1.001987\n",
      "epoch 156, total batch 624529, train loss 0.930983, val loss 0.993071\n",
      "epoch 157, total batch 628314, train loss 0.922460, val loss 0.982818\n",
      "epoch 158, total batch 632099, train loss 0.911253, val loss 0.985533\n",
      "epoch 159, total batch 635884, train loss 0.914856, val loss 0.970242\n",
      "epoch 160, total batch 639669, train loss 0.896867, val loss 0.962772\n",
      "epoch 161, total batch 643454, train loss 0.897442, val loss 0.962915\n",
      "epoch 162, total batch 647239, train loss 0.889505, val loss 0.950119\n",
      "epoch 163, total batch 651024, train loss 0.874793, val loss 0.943762\n",
      "epoch 164, total batch 654809, train loss 0.873282, val loss 0.934879\n",
      "epoch 165, total batch 658594, train loss 0.866862, val loss 0.928678\n",
      "epoch 166, total batch 662379, train loss 0.864888, val loss 0.924695\n",
      "epoch 167, total batch 666164, train loss 0.858335, val loss 0.921981\n",
      "epoch 168, total batch 669949, train loss 0.843437, val loss 0.909849\n",
      "epoch 169, total batch 673734, train loss 0.843586, val loss 0.903800\n",
      "epoch 170, total batch 677519, train loss 0.837898, val loss 0.899161\n",
      "epoch 171, total batch 681304, train loss 0.842042, val loss 0.899224\n",
      "epoch 172, total batch 685089, train loss 0.823896, val loss 0.884304\n",
      "epoch 173, total batch 688874, train loss 0.813273, val loss 0.879888\n",
      "epoch 174, total batch 692659, train loss 0.817869, val loss 0.878645\n",
      "epoch 175, total batch 696444, train loss 0.806847, val loss 0.867903\n",
      "epoch 176, total batch 700229, train loss 0.802063, val loss 0.859582\n",
      "epoch 177, total batch 704014, train loss 0.798503, val loss 0.859430\n",
      "epoch 178, total batch 707799, train loss 0.797027, val loss 0.853133\n",
      "epoch 179, total batch 711584, train loss 0.785643, val loss 0.849568\n",
      "epoch 180, total batch 715369, train loss 0.784948, val loss 0.842452\n",
      "epoch 181, total batch 719154, train loss 0.784017, val loss 0.831735\n",
      "epoch 182, total batch 722939, train loss 0.774748, val loss 0.831668\n",
      "epoch 183, total batch 726724, train loss 0.765324, val loss 0.823885\n",
      "epoch 184, total batch 730509, train loss 0.764411, val loss 0.818545\n",
      "epoch 185, total batch 734294, train loss 0.759993, val loss 0.813153\n",
      "epoch 186, total batch 738079, train loss 0.754086, val loss 0.809021\n",
      "epoch 187, total batch 741864, train loss 0.740241, val loss 0.800439\n",
      "epoch 188, total batch 745649, train loss 0.735145, val loss 0.796853\n",
      "epoch 189, total batch 749434, train loss 0.742018, val loss 0.792548\n",
      "epoch 190, total batch 753219, train loss 0.724393, val loss 0.790934\n",
      "epoch 191, total batch 757004, train loss 0.726504, val loss 0.785744\n",
      "epoch 192, total batch 760789, train loss 0.722762, val loss 0.782237\n",
      "epoch 193, total batch 764574, train loss 0.718582, val loss 0.768984\n",
      "epoch 194, total batch 768359, train loss 0.721163, val loss 0.773169\n",
      "epoch 195, total batch 772144, train loss 0.706376, val loss 0.766127\n",
      "epoch 196, total batch 775929, train loss 0.704333, val loss 0.757583\n",
      "epoch 197, total batch 779714, train loss 0.695768, val loss 0.753425\n",
      "epoch 198, total batch 783499, train loss 0.692356, val loss 0.748917\n",
      "epoch 199, total batch 787284, train loss 0.692909, val loss 0.750107\n",
      "epoch 200, total batch 791069, train loss 0.684776, val loss 0.738608\n",
      "epoch 201, total batch 794854, train loss 0.675706, val loss 0.731707\n",
      "epoch 202, total batch 798639, train loss 0.676059, val loss 0.729632\n",
      "epoch 203, total batch 802424, train loss 0.669522, val loss 0.729682\n",
      "epoch 204, total batch 806209, train loss 0.666360, val loss 0.716819\n",
      "epoch 205, total batch 809994, train loss 0.663448, val loss 0.716808\n",
      "epoch 206, total batch 813779, train loss 0.664307, val loss 0.717402\n",
      "epoch 207, total batch 817564, train loss 0.664081, val loss 0.706768\n",
      "epoch 208, total batch 821349, train loss 0.649256, val loss 0.700051\n",
      "epoch 209, total batch 825134, train loss 0.649358, val loss 0.701064\n",
      "epoch 210, total batch 828919, train loss 0.645989, val loss 0.691638\n",
      "epoch 211, total batch 832704, train loss 0.640313, val loss 0.689491\n",
      "epoch 212, total batch 836489, train loss 0.640655, val loss 0.683561\n",
      "epoch 213, total batch 840274, train loss 0.628446, val loss 0.687079\n",
      "epoch 214, total batch 844059, train loss 0.630491, val loss 0.677452\n",
      "epoch 215, total batch 847844, train loss 0.627452, val loss 0.673729\n",
      "epoch 216, total batch 851629, train loss 0.621781, val loss 0.665208\n",
      "epoch 217, total batch 855414, train loss 0.619434, val loss 0.664371\n",
      "epoch 218, total batch 859199, train loss 0.611052, val loss 0.659631\n",
      "epoch 219, total batch 862984, train loss 0.607892, val loss 0.655185\n",
      "epoch 220, total batch 866769, train loss 0.606476, val loss 0.652656\n",
      "epoch 221, total batch 870554, train loss 0.602781, val loss 0.649884\n",
      "epoch 222, total batch 874339, train loss 0.602334, val loss 0.645514\n",
      "epoch 223, total batch 878124, train loss 0.597164, val loss 0.644364\n",
      "epoch 224, total batch 881909, train loss 0.594722, val loss 0.635501\n",
      "epoch 225, total batch 885694, train loss 0.588730, val loss 0.636597\n",
      "epoch 226, total batch 889479, train loss 0.584741, val loss 0.629561\n",
      "epoch 227, total batch 893264, train loss 0.583316, val loss 0.624337\n",
      "epoch 228, total batch 897049, train loss 0.573893, val loss 0.622342\n",
      "epoch 229, total batch 900834, train loss 0.572752, val loss 0.619025\n",
      "epoch 230, total batch 904619, train loss 0.574175, val loss 0.621194\n",
      "epoch 231, total batch 908404, train loss 0.574168, val loss 0.616047\n",
      "epoch 232, total batch 912189, train loss 0.570940, val loss 0.609708\n",
      "epoch 233, total batch 915974, train loss 0.563270, val loss 0.607236\n",
      "epoch 234, total batch 919759, train loss 0.560744, val loss 0.604019\n",
      "epoch 235, total batch 923544, train loss 0.559725, val loss 0.602183\n",
      "epoch 236, total batch 927329, train loss 0.552287, val loss 0.597640\n",
      "epoch 237, total batch 931114, train loss 0.549659, val loss 0.596033\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 238, total batch 934899, train loss 0.551115, val loss 0.593189\n",
      "epoch 239, total batch 938684, train loss 0.546808, val loss 0.587387\n",
      "epoch 240, total batch 942469, train loss 0.549328, val loss 0.581027\n",
      "epoch 241, total batch 946254, train loss 0.544013, val loss 0.587543\n",
      "epoch 242, total batch 950039, train loss 0.537435, val loss 0.577719\n",
      "epoch 243, total batch 953824, train loss 0.533065, val loss 0.573256\n",
      "epoch 244, total batch 957609, train loss 0.533935, val loss 0.570471\n",
      "epoch 245, total batch 961394, train loss 0.525175, val loss 0.567777\n",
      "epoch 246, total batch 965179, train loss 0.526497, val loss 0.563410\n",
      "epoch 247, total batch 968964, train loss 0.522519, val loss 0.563661\n",
      "epoch 248, total batch 972749, train loss 0.524762, val loss 0.557799\n",
      "epoch 249, total batch 976534, train loss 0.516681, val loss 0.553152\n",
      "epoch 250, total batch 980319, train loss 0.515809, val loss 0.555097\n",
      "epoch 251, total batch 984104, train loss 0.514241, val loss 0.553345\n",
      "epoch 252, total batch 987889, train loss 0.510950, val loss 0.549520\n",
      "epoch 253, total batch 991674, train loss 0.509102, val loss 0.545327\n",
      "epoch 254, total batch 995459, train loss 0.507389, val loss 0.543509\n",
      "epoch 255, total batch 999244, train loss 0.500596, val loss 0.538115\n",
      "epoch 256, total batch 1003029, train loss 0.499006, val loss 0.538157\n",
      "epoch 257, total batch 1006814, train loss 0.496704, val loss 0.531489\n",
      "epoch 258, total batch 1010599, train loss 0.491965, val loss 0.532049\n",
      "epoch 259, total batch 1014384, train loss 0.488710, val loss 0.527640\n",
      "epoch 260, total batch 1018169, train loss 0.488549, val loss 0.528422\n",
      "epoch 261, total batch 1021954, train loss 0.484543, val loss 0.523342\n",
      "epoch 262, total batch 1025739, train loss 0.484014, val loss 0.521505\n",
      "epoch 263, total batch 1029524, train loss 0.482592, val loss 0.516458\n",
      "epoch 264, total batch 1033309, train loss 0.477850, val loss 0.517922\n",
      "epoch 265, total batch 1037094, train loss 0.479158, val loss 0.513596\n",
      "epoch 266, total batch 1040879, train loss 0.473254, val loss 0.511974\n",
      "epoch 267, total batch 1044664, train loss 0.474173, val loss 0.510327\n",
      "epoch 268, total batch 1048449, train loss 0.471403, val loss 0.511037\n",
      "epoch 269, total batch 1052234, train loss 0.470048, val loss 0.508055\n",
      "epoch 270, total batch 1056019, train loss 0.467497, val loss 0.504409\n",
      "epoch 271, total batch 1059804, train loss 0.462545, val loss 0.498604\n",
      "epoch 272, total batch 1063589, train loss 0.454928, val loss 0.494058\n",
      "epoch 273, total batch 1067374, train loss 0.453578, val loss 0.493985\n",
      "epoch 274, total batch 1071159, train loss 0.459462, val loss 0.487103\n",
      "epoch 275, total batch 1074944, train loss 0.458622, val loss 0.486106\n",
      "epoch 276, total batch 1078729, train loss 0.454375, val loss 0.485637\n",
      "epoch 277, total batch 1082514, train loss 0.446994, val loss 0.486121\n",
      "epoch 278, total batch 1086299, train loss 0.448480, val loss 0.483342\n",
      "epoch 279, total batch 1090084, train loss 0.445159, val loss 0.483962\n",
      "epoch 280, total batch 1093869, train loss 0.448040, val loss 0.477070\n",
      "epoch 281, total batch 1097654, train loss 0.440158, val loss 0.479152\n",
      "epoch 282, total batch 1101439, train loss 0.441191, val loss 0.474144\n",
      "epoch 283, total batch 1105224, train loss 0.436619, val loss 0.473336\n",
      "epoch 284, total batch 1109009, train loss 0.432618, val loss 0.468771\n",
      "epoch 285, total batch 1112794, train loss 0.432668, val loss 0.468055\n",
      "epoch 286, total batch 1116579, train loss 0.434768, val loss 0.462894\n",
      "epoch 287, total batch 1120364, train loss 0.431473, val loss 0.463231\n",
      "epoch 288, total batch 1124149, train loss 0.427630, val loss 0.459641\n",
      "epoch 289, total batch 1127934, train loss 0.426441, val loss 0.457201\n",
      "epoch 290, total batch 1131719, train loss 0.417771, val loss 0.453063\n",
      "epoch 291, total batch 1135504, train loss 0.421735, val loss 0.453491\n",
      "epoch 292, total batch 1139289, train loss 0.418760, val loss 0.452180\n",
      "epoch 293, total batch 1143074, train loss 0.420984, val loss 0.449351\n",
      "epoch 294, total batch 1146859, train loss 0.417982, val loss 0.451807\n",
      "epoch 295, total batch 1150644, train loss 0.414091, val loss 0.449286\n",
      "epoch 296, total batch 1154429, train loss 0.409521, val loss 0.443502\n",
      "epoch 297, total batch 1158214, train loss 0.409114, val loss 0.440134\n",
      "epoch 298, total batch 1161999, train loss 0.407578, val loss 0.443705\n",
      "epoch 299, total batch 1165784, train loss 0.403748, val loss 0.437484\n",
      "epoch 300, total batch 1169569, train loss 0.406574, val loss 0.434843\n",
      "epoch 301, total batch 1173354, train loss 0.399475, val loss 0.430153\n",
      "epoch 302, total batch 1177139, train loss 0.397200, val loss 0.436598\n",
      "epoch 303, total batch 1180924, train loss 0.399570, val loss 0.431053\n",
      "epoch 304, total batch 1184709, train loss 0.394055, val loss 0.433254\n",
      "epoch 305, total batch 1188494, train loss 0.397006, val loss 0.428764\n",
      "epoch 306, total batch 1192279, train loss 0.394631, val loss 0.426682\n",
      "epoch 307, total batch 1196064, train loss 0.392334, val loss 0.424509\n",
      "epoch 308, total batch 1199849, train loss 0.389840, val loss 0.420182\n",
      "epoch 309, total batch 1203634, train loss 0.389637, val loss 0.419662\n",
      "epoch 310, total batch 1207419, train loss 0.385877, val loss 0.420442\n",
      "epoch 311, total batch 1211204, train loss 0.389645, val loss 0.417425\n",
      "epoch 312, total batch 1214989, train loss 0.386483, val loss 0.414185\n",
      "epoch 313, total batch 1218774, train loss 0.380453, val loss 0.416929\n",
      "epoch 314, total batch 1222559, train loss 0.380113, val loss 0.410514\n",
      "epoch 315, total batch 1226344, train loss 0.374954, val loss 0.412235\n",
      "epoch 316, total batch 1230129, train loss 0.378659, val loss 0.410645\n",
      "epoch 317, total batch 1233914, train loss 0.374491, val loss 0.406945\n",
      "epoch 318, total batch 1237699, train loss 0.369950, val loss 0.403801\n",
      "epoch 319, total batch 1241484, train loss 0.375789, val loss 0.404462\n",
      "epoch 320, total batch 1245269, train loss 0.368521, val loss 0.399684\n",
      "epoch 321, total batch 1249054, train loss 0.366853, val loss 0.395553\n",
      "epoch 322, total batch 1252839, train loss 0.365614, val loss 0.397810\n",
      "epoch 323, total batch 1256624, train loss 0.368149, val loss 0.395300\n",
      "epoch 324, total batch 1260409, train loss 0.366360, val loss 0.393801\n",
      "epoch 325, total batch 1264194, train loss 0.363900, val loss 0.393765\n",
      "epoch 326, total batch 1267979, train loss 0.359412, val loss 0.386282\n",
      "epoch 327, total batch 1271764, train loss 0.357415, val loss 0.391835\n",
      "epoch 328, total batch 1275549, train loss 0.358563, val loss 0.388489\n",
      "epoch 329, total batch 1279334, train loss 0.352442, val loss 0.384247\n",
      "epoch 330, total batch 1283119, train loss 0.357006, val loss 0.383969\n",
      "epoch 331, total batch 1286904, train loss 0.357895, val loss 0.382486\n",
      "epoch 332, total batch 1290689, train loss 0.352021, val loss 0.379413\n",
      "epoch 333, total batch 1294474, train loss 0.351359, val loss 0.380464\n",
      "epoch 334, total batch 1298259, train loss 0.350624, val loss 0.376165\n",
      "epoch 335, total batch 1302044, train loss 0.348858, val loss 0.378062\n",
      "epoch 336, total batch 1305829, train loss 0.349767, val loss 0.375472\n",
      "epoch 337, total batch 1309614, train loss 0.351261, val loss 0.374692\n",
      "epoch 338, total batch 1313399, train loss 0.345269, val loss 0.374482\n",
      "epoch 339, total batch 1317184, train loss 0.341857, val loss 0.371876\n",
      "epoch 340, total batch 1320969, train loss 0.341397, val loss 0.370359\n",
      "epoch 341, total batch 1324754, train loss 0.342577, val loss 0.369294\n",
      "epoch 342, total batch 1328539, train loss 0.338266, val loss 0.363272\n",
      "epoch 343, total batch 1332324, train loss 0.336489, val loss 0.367768\n",
      "epoch 344, total batch 1336109, train loss 0.335145, val loss 0.367126\n",
      "epoch 345, total batch 1339894, train loss 0.333557, val loss 0.359157\n",
      "epoch 346, total batch 1343679, train loss 0.334845, val loss 0.361685\n",
      "epoch 347, total batch 1347464, train loss 0.334244, val loss 0.357872\n",
      "epoch 348, total batch 1351249, train loss 0.329804, val loss 0.361339\n",
      "epoch 349, total batch 1355034, train loss 0.327604, val loss 0.354570\n",
      "epoch 350, total batch 1358819, train loss 0.326304, val loss 0.357702\n",
      "epoch 351, total batch 1362604, train loss 0.328073, val loss 0.354496\n",
      "epoch 352, total batch 1366389, train loss 0.328960, val loss 0.354872\n",
      "epoch 353, total batch 1370174, train loss 0.324969, val loss 0.353443\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 354, total batch 1373959, train loss 0.323804, val loss 0.349504\n",
      "epoch 355, total batch 1377744, train loss 0.325351, val loss 0.348757\n",
      "epoch 356, total batch 1381529, train loss 0.322968, val loss 0.351647\n",
      "epoch 357, total batch 1385314, train loss 0.317750, val loss 0.349809\n",
      "epoch 358, total batch 1389099, train loss 0.320367, val loss 0.345240\n",
      "epoch 359, total batch 1392884, train loss 0.317634, val loss 0.343972\n",
      "epoch 360, total batch 1396669, train loss 0.316336, val loss 0.338748\n",
      "epoch 361, total batch 1400454, train loss 0.317305, val loss 0.340585\n",
      "epoch 362, total batch 1404239, train loss 0.313842, val loss 0.338703\n",
      "epoch 363, total batch 1408024, train loss 0.311719, val loss 0.340664\n",
      "epoch 364, total batch 1411809, train loss 0.311886, val loss 0.341744\n",
      "epoch 365, total batch 1415594, train loss 0.316481, val loss 0.340289\n",
      "epoch 366, total batch 1419379, train loss 0.315889, val loss 0.338442\n",
      "epoch 367, total batch 1423164, train loss 0.311069, val loss 0.334905\n",
      "epoch 368, total batch 1426949, train loss 0.309543, val loss 0.333929\n",
      "epoch 369, total batch 1430734, train loss 0.308403, val loss 0.332794\n",
      "epoch 370, total batch 1434519, train loss 0.310868, val loss 0.332328\n",
      "epoch 371, total batch 1438304, train loss 0.305259, val loss 0.334195\n",
      "epoch 372, total batch 1442089, train loss 0.305121, val loss 0.328791\n",
      "epoch 373, total batch 1445874, train loss 0.303469, val loss 0.328281\n",
      "epoch 374, total batch 1449659, train loss 0.302971, val loss 0.329724\n",
      "epoch 375, total batch 1453444, train loss 0.301468, val loss 0.324493\n",
      "epoch 376, total batch 1457229, train loss 0.298879, val loss 0.323110\n",
      "epoch 377, total batch 1461014, train loss 0.294754, val loss 0.322564\n",
      "epoch 378, total batch 1464799, train loss 0.297438, val loss 0.320221\n",
      "epoch 379, total batch 1468584, train loss 0.296679, val loss 0.321817\n",
      "epoch 380, total batch 1472369, train loss 0.297495, val loss 0.322389\n",
      "epoch 381, total batch 1476154, train loss 0.293237, val loss 0.318009\n",
      "epoch 382, total batch 1479939, train loss 0.294387, val loss 0.317903\n",
      "epoch 383, total batch 1483724, train loss 0.290086, val loss 0.318746\n",
      "epoch 384, total batch 1487509, train loss 0.290008, val loss 0.316436\n",
      "epoch 385, total batch 1491294, train loss 0.289839, val loss 0.314255\n",
      "epoch 386, total batch 1495079, train loss 0.286676, val loss 0.310753\n",
      "epoch 387, total batch 1498864, train loss 0.287388, val loss 0.311491\n",
      "epoch 388, total batch 1502649, train loss 0.284768, val loss 0.308621\n",
      "epoch 389, total batch 1506434, train loss 0.283743, val loss 0.310167\n",
      "epoch 390, total batch 1510219, train loss 0.287812, val loss 0.313134\n",
      "epoch 391, total batch 1514004, train loss 0.285448, val loss 0.311446\n",
      "epoch 392, total batch 1517789, train loss 0.283803, val loss 0.308715\n",
      "epoch 393, total batch 1521574, train loss 0.280527, val loss 0.303322\n",
      "epoch 394, total batch 1525359, train loss 0.283828, val loss 0.304152\n",
      "epoch 395, total batch 1529144, train loss 0.282218, val loss 0.304100\n",
      "epoch 396, total batch 1532929, train loss 0.277430, val loss 0.302299\n",
      "epoch 397, total batch 1536714, train loss 0.274240, val loss 0.300935\n",
      "epoch 398, total batch 1540499, train loss 0.277766, val loss 0.303210\n",
      "epoch 399, total batch 1544284, train loss 0.278228, val loss 0.299234\n",
      "epoch 400, total batch 1548069, train loss 0.279185, val loss 0.298042\n",
      "epoch 401, total batch 1551854, train loss 0.273657, val loss 0.299217\n",
      "epoch 402, total batch 1555639, train loss 0.273769, val loss 0.295722\n",
      "epoch 403, total batch 1559424, train loss 0.272983, val loss 0.294608\n",
      "epoch 404, total batch 1563209, train loss 0.270462, val loss 0.297599\n",
      "epoch 405, total batch 1566994, train loss 0.273331, val loss 0.294017\n",
      "epoch 406, total batch 1570779, train loss 0.272217, val loss 0.292581\n",
      "epoch 407, total batch 1574564, train loss 0.270680, val loss 0.290717\n",
      "epoch 408, total batch 1578349, train loss 0.268134, val loss 0.291974\n",
      "epoch 409, total batch 1582134, train loss 0.268032, val loss 0.288314\n",
      "epoch 410, total batch 1585919, train loss 0.265710, val loss 0.292956\n",
      "epoch 411, total batch 1589704, train loss 0.263143, val loss 0.290069\n",
      "epoch 412, total batch 1593489, train loss 0.268287, val loss 0.291292\n",
      "epoch 413, total batch 1597274, train loss 0.265841, val loss 0.288406\n",
      "epoch 414, total batch 1601059, train loss 0.262625, val loss 0.288872\n",
      "epoch 415, total batch 1604844, train loss 0.264722, val loss 0.288211\n",
      "epoch 416, total batch 1608629, train loss 0.265198, val loss 0.284688\n",
      "epoch 417, total batch 1612414, train loss 0.258816, val loss 0.278941\n",
      "epoch 418, total batch 1616199, train loss 0.261246, val loss 0.283564\n",
      "epoch 419, total batch 1619984, train loss 0.259240, val loss 0.282367\n",
      "epoch 420, total batch 1623769, train loss 0.255648, val loss 0.281060\n",
      "epoch 421, total batch 1627554, train loss 0.258387, val loss 0.276475\n",
      "epoch 422, total batch 1631339, train loss 0.255967, val loss 0.276943\n",
      "epoch 423, total batch 1635124, train loss 0.255333, val loss 0.274613\n",
      "epoch 424, total batch 1638909, train loss 0.254817, val loss 0.273755\n",
      "epoch 425, total batch 1642694, train loss 0.256410, val loss 0.271853\n",
      "epoch 426, total batch 1646479, train loss 0.257525, val loss 0.273828\n",
      "epoch 427, total batch 1650264, train loss 0.254415, val loss 0.275066\n",
      "epoch 428, total batch 1654049, train loss 0.256088, val loss 0.275187\n",
      "epoch 429, total batch 1657834, train loss 0.248654, val loss 0.273950\n",
      "epoch 430, total batch 1661619, train loss 0.255652, val loss 0.273428\n",
      "epoch 431, total batch 1665404, train loss 0.248204, val loss 0.271486\n",
      "epoch 432, total batch 1669189, train loss 0.254071, val loss 0.273090\n",
      "epoch 433, total batch 1672974, train loss 0.251005, val loss 0.274699\n",
      "epoch 434, total batch 1676759, train loss 0.249889, val loss 0.271292\n",
      "epoch 435, total batch 1680544, train loss 0.246350, val loss 0.268693\n",
      "epoch 436, total batch 1684329, train loss 0.247011, val loss 0.266684\n",
      "epoch 437, total batch 1688114, train loss 0.243916, val loss 0.267003\n",
      "epoch 438, total batch 1691899, train loss 0.242591, val loss 0.267457\n",
      "epoch 439, total batch 1695684, train loss 0.244341, val loss 0.265291\n",
      "epoch 440, total batch 1699469, train loss 0.241556, val loss 0.264633\n",
      "epoch 441, total batch 1703254, train loss 0.240271, val loss 0.263970\n",
      "epoch 442, total batch 1707039, train loss 0.241986, val loss 0.260913\n",
      "epoch 443, total batch 1710824, train loss 0.241753, val loss 0.263138\n",
      "epoch 444, total batch 1714609, train loss 0.238162, val loss 0.258121\n",
      "epoch 445, total batch 1718394, train loss 0.240450, val loss 0.259533\n",
      "epoch 446, total batch 1722179, train loss 0.236306, val loss 0.259381\n",
      "epoch 447, total batch 1725964, train loss 0.235189, val loss 0.259612\n",
      "epoch 448, total batch 1729749, train loss 0.241459, val loss 0.257640\n",
      "epoch 449, total batch 1733534, train loss 0.235942, val loss 0.259011\n",
      "epoch 450, total batch 1737319, train loss 0.239687, val loss 0.259020\n",
      "epoch 451, total batch 1741104, train loss 0.236653, val loss 0.256411\n",
      "epoch 452, total batch 1744889, train loss 0.236551, val loss 0.255583\n",
      "epoch 453, total batch 1748674, train loss 0.238277, val loss 0.255196\n",
      "epoch 454, total batch 1752459, train loss 0.234345, val loss 0.252092\n",
      "epoch 455, total batch 1756244, train loss 0.233568, val loss 0.256175\n",
      "epoch 456, total batch 1760029, train loss 0.230808, val loss 0.252121\n",
      "epoch 457, total batch 1763814, train loss 0.235698, val loss 0.252105\n",
      "epoch 458, total batch 1767599, train loss 0.232254, val loss 0.250895\n",
      "epoch 459, total batch 1771384, train loss 0.228601, val loss 0.248961\n",
      "epoch 460, total batch 1775169, train loss 0.230482, val loss 0.244838\n",
      "epoch 461, total batch 1778954, train loss 0.228887, val loss 0.243399\n",
      "epoch 462, total batch 1782739, train loss 0.229421, val loss 0.247169\n",
      "epoch 463, total batch 1786524, train loss 0.229172, val loss 0.247496\n",
      "epoch 464, total batch 1790309, train loss 0.223932, val loss 0.246309\n",
      "epoch 465, total batch 1794094, train loss 0.224854, val loss 0.242638\n",
      "epoch 466, total batch 1797879, train loss 0.227664, val loss 0.245365\n",
      "epoch 467, total batch 1801664, train loss 0.224713, val loss 0.244436\n",
      "epoch 468, total batch 1805449, train loss 0.224911, val loss 0.242985\n",
      "epoch 469, total batch 1809234, train loss 0.223386, val loss 0.240747\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 470, total batch 1813019, train loss 0.220846, val loss 0.244321\n",
      "epoch 471, total batch 1816804, train loss 0.219363, val loss 0.239567\n",
      "epoch 472, total batch 1820589, train loss 0.218047, val loss 0.239677\n",
      "epoch 473, total batch 1824374, train loss 0.222755, val loss 0.237155\n",
      "epoch 474, total batch 1828159, train loss 0.222048, val loss 0.239926\n",
      "epoch 475, total batch 1831944, train loss 0.220435, val loss 0.238174\n",
      "epoch 476, total batch 1835729, train loss 0.217247, val loss 0.237537\n",
      "epoch 477, total batch 1839514, train loss 0.219757, val loss 0.236280\n",
      "epoch 478, total batch 1843299, train loss 0.213726, val loss 0.236067\n",
      "epoch 479, total batch 1847084, train loss 0.214814, val loss 0.230011\n",
      "epoch 480, total batch 1850869, train loss 0.217010, val loss 0.234016\n",
      "epoch 481, total batch 1854654, train loss 0.214691, val loss 0.232747\n",
      "epoch 482, total batch 1858439, train loss 0.214700, val loss 0.232798\n",
      "epoch 483, total batch 1862224, train loss 0.216038, val loss 0.230204\n",
      "epoch 484, total batch 1866009, train loss 0.216376, val loss 0.230604\n",
      "epoch 485, total batch 1869794, train loss 0.212690, val loss 0.233236\n",
      "epoch 486, total batch 1873579, train loss 0.210286, val loss 0.233520\n",
      "epoch 487, total batch 1877364, train loss 0.213618, val loss 0.229512\n",
      "epoch 488, total batch 1881149, train loss 0.209884, val loss 0.228651\n",
      "epoch 489, total batch 1884934, train loss 0.207168, val loss 0.226483\n",
      "epoch 490, total batch 1888719, train loss 0.211910, val loss 0.229990\n",
      "epoch 491, total batch 1892504, train loss 0.209721, val loss 0.228765\n",
      "epoch 492, total batch 1896289, train loss 0.207178, val loss 0.224474\n",
      "epoch 493, total batch 1900074, train loss 0.207304, val loss 0.225606\n",
      "epoch 494, total batch 1903859, train loss 0.209121, val loss 0.231178\n",
      "epoch 495, total batch 1907644, train loss 0.208782, val loss 0.225370\n",
      "epoch 496, total batch 1911429, train loss 0.208007, val loss 0.224810\n",
      "epoch 497, total batch 1915214, train loss 0.206691, val loss 0.222952\n",
      "epoch 498, total batch 1918999, train loss 0.205180, val loss 0.222757\n",
      "epoch 499, total batch 1922784, train loss 0.204490, val loss 0.221926\n",
      "epoch 500, total batch 1926569, train loss 0.205564, val loss 0.221980\n"
     ]
    }
   ],
   "source": [
    "embed_weights, nce_weights = model.train(X, Y, X_val, Y_val,\n",
    "                                         graph_params['epoch'],\n",
    "                                         graph_params['batch_size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vector_embed = word2vec.Word2Vec(embed_weights, dictionary)\n",
    "word_vector_nce = word2vec.Word2Vec(nce_weights, dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding layer: 8 closest words to: 'mahathir'\n",
      "['mahahir', 'yusri', 'shaeraine', 'hamidi', 'xavier', 'mentaloon', 'subramaniam', 'ignatius'] \n",
      "\n",
      "NCE layer: 8 closest words to: 'mahathir'\n",
      "['subramaniam', 'kyi', 'asyraf', 'lee', 'kerasnya', 'obama', 'mahahir', 'mpa']\n"
     ]
    }
   ],
   "source": [
    "word = 'mahathir'\n",
    "print(\"Embedding layer: 8 closest words to: '%s'\"%(word))\n",
    "print(word_vector_embed.n_closest(word=word, num_closest=8, metric='cosine'), '\\n')\n",
    "print(\"NCE layer: 8 closest words to: '%s'\"%(word))\n",
    "print(word_vector_nce.n_closest(word=word, num_closest=8, metric='cosine'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['kerajaan', 'anwar', 'keadaan', 'kpm', 'sultan']\n"
     ]
    }
   ],
   "source": [
    "print(word_vector_nce.analogy('mahathir', 'anwar', 'kerajaan', 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['anwar', 'kerajaan', 'expo', 'harapansepanjang', 'perogol']\n"
     ]
    }
   ],
   "source": [
    "print(word_vector_embed.analogy('mahathir', 'anwar', 'kerajaan', 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "passage = [rev_dictionary[i] for i in word_array[12200:12300]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " sangat dominan sedangkan mca ada masalah mic dan gerakan ada masalah ini yang lemahkan bnnamun nature politik memang begitu tapi boleh settle down lepas prucuma bimbang orang malaysia sangat emosi dengan apa yang berlaku maka ia tidak sihatazizudin politik kena rasional jadi kita patut ke arah politik matangnamun hakikatnya melihat pada isu kaum ia berlaku di semua negarakena rasionalkan politik perkauman itu cuma di malaysia kita masih baik dan tidak berperang sesama sendirilihat faktor ekonomi mempengaruhi politik jadi fenomena arab spring tidak akan berlaku di malaysiamoderator adakah bn boleh reformmaszlee kalau kita lihat bn dan umno ada dinamik berevolusi ketika\n"
     ]
    }
   ],
   "source": [
    "readable = ''\n",
    "for word in passage:\n",
    "    if word == '\"':\n",
    "        readable += word\n",
    "    elif word in ['?', '!', '.', ',']:\n",
    "        readable += word + ' '\n",
    "    else: \n",
    "        readable += ' ' + word\n",
    "print(readable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = word2vec.build_training_set(word_array[(12200-2):(12300+2)])\n",
    "y_hat = model.sess.run(model.logits,feed_dict={model.X:x})\n",
    "passage_predict = [rev_dictionary[i] for i in y_hat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " sangat dominan sejarah pas ada masalah dalaman dan gerakan ada masalah ini yang lemahkan dibutuhkan nature politik memang begitu tapi boleh tukar kejayaan lepas saya bimbang orang malaysia menentang baik dengan apa yang berlaku kini ia tidak semestinya politik kena segera jadi kita patut ke arah politik kita apabila melihat pada isu kaum ia berlaku di semua tempat jahat politik kebencian itu cuma di rumah kita masih marah dan tidak sihat sesama dipengaruhi faktor ekonomi politik politik jadi fenomena arab saudi tidak akan berlaku di sini jentera bn boleh jadi kalau kita minta bn dan umno ada dinamik beberapa ketika\n"
     ]
    }
   ],
   "source": [
    "readable = ''\n",
    "for word in passage_predict:\n",
    "    if word == '\"':\n",
    "        readable += word\n",
    "    elif word in ['?', '!', '.', ',']:\n",
    "        readable += word + ' '\n",
    "    else: \n",
    "        readable += ' ' + word\n",
    "print(readable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('word2vec-512.p', 'wb') as fopen:\n",
    "    pickle.dump({'dictionary':dictionary,'rev_dictionary':rev_dictionary,\n",
    "                 'embed_weights':embed_weights,'nce_weights':nce_weights}, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
